import pandas as pd

# functions=pd.read_csv('datasets/funcom_tokenized/test/functions.test', header=None, sep='\t')
# comments=pd.read_csv('datasets/funcom_tokenized/test/comments.test', header=None, sep='\t')
# funcoms = pd.concat((functions[1], comments[1]), axis=1)
# funcoms.columns = ['code', 'comment']
# print(funcoms.head())
# funcoms.to_csv('datasets/funcom_tokenized/test/funcoms.csv')

#
# functions=pd.read_csv('datasets/fold_1/train/train.token.code', header=None, sep='\n')
# comments=pd.read_csv('datasets/fold_1/train/train.token.nl', header=None, sep='\n')
# funcoms = pd.concat((functions, comments), axis=1)
# funcoms.columns = ['code', 'comment']
# print(funcoms.head())
# funcoms.to_csv('datasets/fold_1/train/funcoms.csv')

# data = pd.read_csv('datasets/funcom_tokenized/train/funcoms.csv')
# data.dropna(axis=0, how='any', inplace=True)
# data = data.sample(100000)
# print(data)
# data.to_csv('datasets/funcom_tokenized/train/funcoms-100000.csv')

# dataset = pd.read_csv('../datasets/bug-report-title/valid-cased.csv', low_memory=False)
# # train_dataset.dropna(axis=0, how='any', inplace=True)
# dataset['body'] = dataset['body'].str.lower()
# dataset['title'] = dataset['title'].str.lower()
# dataset.to_csv('../datasets/bug-report-title/valid.csv')


# src=pd.read_csv('../datasets/question_title/src-train.txt', header=None, sep='\n')
# target=pd.read_csv('../datasets/question_title/tgt-train.txt', header=None, sep='\n')
# data = pd.concat((src[0], target[0]), axis=1)
# data.columns = ['src', 'target']
# print(data.head())
# valid = data.sample(3000)
# train = data.drop(labels=valid.index)
# train.to_csv('../datasets/question_title/train.csv')
# valid.to_csv('../datasets/question_title/valid.csv')
# print(train)
# print(valid)
# print(data)
# src=pd.read_csv('../datasets/commit_message/cleaned.test.diff', header=None, sep='\n')
# target=pd.read_csv('../datasets/commit_message/cleaned.test.msg', header=None, sep='\n')
# data = pd.concat((src[0], target[0]), axis=1)
# data.columns = ['src', 'target']
# # lower case
# # data['src'] = data['src'].str.lower()
# # data['target'] = data['target'].str.lower()
# data.to_csv('../original-model-hypothesis/commit-message/refs.csv')
# print(data)

# hypothesis = []
# cache = []
# with open('../original-model-hypothesis/commit-message-opennmt/commit-nmt_itape_step_25000_candidate100_minlen2.txt') as f:
#     idx = 0
#     line = f.readline()
#     while line:
#         cache.append(line.strip('\n').lower())
#         idx += 1
#         if idx % 100 == 0:
#             hypothesis.append(cache)
#             cache = []
#         line = f.readline()
#
# hypothesis = pd.DataFrame(hypothesis)
# # hypothesis.columns = ['title']
# print(hypothesis)
# hypothesis.to_csv('../original-model-hypothesis/commit-message-opennmt/commit-nmt_itape_step_25000_candidate100_minlen2.csv')

# import os
# hypothesis = []
#
# hyp_dir = '../original-model-hypothesis/so-question-title/result-bs4/beam_summary/'
# files = os.listdir(hyp_dir)
# files.sort()
# for file in files:
#     print(file)
#     with open(hyp_dir + file) as f:
#         cache = []
#         line = f.readline()
#         while line:
#             line = line.strip('\n')
#             line = line[line.index('\t') + 1:]
#             # if line.find('<eos>') != -1:
#             #     line = line[:line.index('<eos>')-1]
#             cache.append(line)
#             line = f.readline()
#         hypothesis.append(cache)
#
# hypothesis = pd.DataFrame(hypothesis)
# # hypothesis.columns = ['target']
# print(hypothesis)
# hypothesis.to_csv('../original-model-hypothesis/so-question-title/hypothesis-witheos-bs-4.csv')

data = pd.read_csv('../original-model-hypothesis/SOTitle-bos-eos-1063-30/predictions-bs-100.csv')
data = data.iloc[:, 1:101].values.tolist()
len = len(data)
for i in range(len):
    for j in range(100):
        data[i][j] = str(data[i][j]).strip().lower()

data = pd.DataFrame(data)
print(data)
data.to_csv('../original-model-hypothesis/SOTitle-bos-eos-1063-30/predictions-bs-100-lower.csv')

# body=pd.read_csv('../datasets/SOTitle/test_body.csv')
# code=pd.read_csv('../datasets/SOTitle/test_code.csv')
# title=pd.read_csv('../datasets/SOTitle/test_title.csv')
# data = pd.concat((body, code), axis=1)
# data.columns = ['body', 'code']
# data['src'] = data.apply(lambda row: ' '.join(row['body'].split()[:256]) + " <code> " + ' '.join(row['code'].split()[:256]), axis=1)
# data['target'] = title
# data = data[['src', 'target']]
# print(data)
# data.to_csv('../datasets/SOTitle/test.csv')
